# -*- coding: utf-8 -*-
"""
数据处理模块
"""

import hashlib
import logging
import re
from datetime import datetime, timedelta
from typing import List, Dict, Any, Optional

from config.settings import DATA_CONFIG


class DataProcessor:
    """数据处理器"""
    
    def __init__(self):
        self.logger = logging.getLogger(__name__)
    
    def process_crawled_data(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """处理爬取的原始数据"""
        try:
            self.logger.info(f"开始处理 {len(raw_data)} 条原始数据")
            
            processed_data = []
            
            for i, item in enumerate(raw_data):
                try:
                    # 数据清洗
                    cleaned_item = self._clean_single_item(item)
                    
                    # 数据验证
                    if self._validate_item(cleaned_item):
                        # 生成数据指纹
                        cleaned_item['data_fingerprint'] = self._generate_fingerprint(cleaned_item)
                        
                        # 添加处理时间戳
                        cleaned_item['crawl_time'] = datetime.now()
                        cleaned_item['created_at'] = datetime.now()
                        cleaned_item['updated_at'] = datetime.now()
                        
                        processed_data.append(cleaned_item)
                    else:
                        self.logger.warning(f"第 {i+1} 条数据验证失败，跳过")
                        
                except Exception as e:
                    self.logger.error(f"处理第 {i+1} 条数据失败: {e}")
                    continue
            
            self.logger.info(f"数据处理完成，有效数据 {len(processed_data)} 条")
            return processed_data
            
        except Exception as e:
            self.logger.error(f"数据处理失败: {e}")
            return []
    
    def _clean_single_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
        """清洗单条数据"""
        cleaned = {}
        
        # 城市
        cleaned['city'] = self._clean_text(item.get('city', '深圳'))
        
        # 联系人
        contact_person = item.get('contact_person', '')
        cleaned['contact_person'] = self._clean_contact_person(contact_person)
        
        # 联系方式
        contact_info = item.get('contact_info', '')
        cleaned['contact_info'] = self._clean_contact_info(contact_info)
        
        # 业态
        business_type = item.get('business_type', '餐饮')
        cleaned['business_type'] = self._clean_business_type(business_type)
        
        # 租金
        acceptable_rent = item.get('acceptable_rent', '面议')
        cleaned['acceptable_rent'] = self._clean_rent_info(acceptable_rent)
        
        # 面积
        building_area = item.get('building_area', '')
        cleaned['building_area'] = self._clean_area_info(building_area)
        
        # 发布日期
        publish_date = item.get('publish_date')
        cleaned['publish_date'] = self._clean_publish_date(publish_date)
        
        # 详情页URL
        detail_url = item.get('detail_url', '')
        cleaned['detail_url'] = self._clean_url(detail_url)
        
        # 数据来源
        cleaned['data_source'] = item.get('data_source', 'pupuwang')
        
        # 数据状态
        cleaned['data_status'] = 'active'
        
        return cleaned
    
    def _clean_text(self, text: Any) -> str:
        """清洗文本"""
        if not text:
            return ""
        
        text = str(text).strip()
        # 移除多余的空白字符
        text = re.sub(r'\s+', ' ', text)
        return text
    
    def _clean_contact_person(self, contact: Any) -> str:
        """清洗联系人信息"""
        contact = self._clean_text(contact)
        
        if not contact:
            return ""
        
        # 移除常见的无用信息
        contact = re.sub(r'[（(][^）)]*[）)]', '', contact)  # 移除括号内容
        contact = re.sub(r'先生|女士|小姐', '', contact)  # 移除称谓
        contact = contact.strip()
        
        # 如果太短或包含数字，可能不是真实姓名
        if len(contact) < 1 or re.search(r'\d', contact):
            return ""
        
        return contact[:10]  # 限制长度
    
    def _clean_contact_info(self, contact_info: Any) -> str:
        """清洗联系方式"""
        contact_info = self._clean_text(contact_info)
        
        if not contact_info:
            return ""
        
        # 提取手机号码
        phone_pattern = r'1[3-9]\d{9}'
        phone_match = re.search(phone_pattern, contact_info)
        
        if phone_match:
            return phone_match.group(0)
        
        # 如果没有找到标准手机号，返回原始信息（可能是座机或其他联系方式）
        return contact_info[:50]  # 限制长度
    
    def _clean_business_type(self, business_type: Any) -> str:
        """清洗业态信息"""
        business_type = self._clean_text(business_type)
        
        # 标准化业态名称
        type_mapping = {
            '餐馆': '餐饮',
            '餐厅': '餐饮',
            '饭店': '餐饮',
            '小吃': '餐饮',
            '快餐': '餐饮',
            '火锅': '餐饮',
            '咖啡': '餐饮',
            '茶饮': '餐饮',
            '娱乐': '娱乐',
            '游戏': '娱乐',
            'KTV': '娱乐',
            '网吧': '娱乐',
            '零售': '零售',
            '服装': '零售',
            '超市': '零售',
            '便利店': '零售'
        }
        
        for key, value in type_mapping.items():
            if key in business_type:
                return value
        
        return business_type if business_type else '餐饮'
    
    def _clean_rent_info(self, rent: Any) -> str:
        """清洗租金信息"""
        rent = self._clean_text(rent)
        
        if not rent:
            return "面议"
        
        # 标准化租金表示
        rent = re.sub(r'租金[：:]?\s*', '', rent)
        
        # 如果包含数字，保留原格式
        if re.search(r'\d', rent):
            return rent
        
        return "面议"
    
    def _clean_area_info(self, area: Any) -> str:
        """清洗面积信息"""
        area = self._clean_text(area)
        
        if not area:
            return ""
        
        # 提取面积数字和单位
        area_pattern = r'(\d+[-~]\d+|\d+)\s*平米?'
        area_match = re.search(area_pattern, area)
        
        if area_match:
            return area_match.group(0)
        
        return area
    
    def _clean_publish_date(self, date: Any) -> Optional[datetime]:
        """清洗发布日期"""
        if isinstance(date, datetime):
            return date
        
        if not date:
            return datetime.now()
        
        # 如果是字符串，尝试解析
        if isinstance(date, str):
            try:
                # 尝试多种日期格式
                formats = [
                    '%Y-%m-%d %H:%M:%S',
                    '%Y-%m-%d',
                    '%Y/%m/%d %H:%M:%S',
                    '%Y/%m/%d'
                ]
                
                for fmt in formats:
                    try:
                        return datetime.strptime(date, fmt)
                    except ValueError:
                        continue
                        
            except Exception:
                pass
        
        return datetime.now()
    
    def _clean_url(self, url: Any) -> str:
        """清洗URL"""
        url = self._clean_text(url)
        
        if not url:
            return ""
        
        # 如果是相对URL，保持原样
        if url.startswith('/'):
            return url
        
        # 如果是完整URL，验证格式
        if url.startswith('http'):
            return url
        
        return ""
    
    def _validate_item(self, item: Dict[str, Any]) -> bool:
        """验证数据项"""
        try:
            # 检查必需字段
            required_fields = ['city', 'business_type']
            
            for field in required_fields:
                if not item.get(field):
                    self.logger.warning(f"缺少必需字段: {field}")
                    return False
            
            # 检查发布日期是否在有效范围内
            publish_date = item.get('publish_date')
            if publish_date:
                cutoff_date = datetime.now() - timedelta(days=DATA_CONFIG['date_range_days'])
                if publish_date < cutoff_date:
                    self.logger.warning(f"数据过期: {publish_date}")
                    return False
            
            # 检查数据质量
            contact_person = item.get('contact_person', '')
            contact_info = item.get('contact_info', '')
            
            # 至少要有联系人或联系方式之一
            if not contact_person and not contact_info:
                self.logger.warning("缺少联系信息")
                return False
            
            return True
            
        except Exception as e:
            self.logger.error(f"数据验证失败: {e}")
            return False
    
    def _generate_fingerprint(self, item: Dict[str, Any]) -> str:
        """生成数据指纹用于去重"""
        try:
            # 使用关键字段生成指纹
            key_fields = [
                item.get('contact_info', ''),
                item.get('contact_person', ''),
                item.get('business_type', ''),
                item.get('city', ''),
                item.get('building_area', ''),
                item.get('acceptable_rent', '')
            ]
            
            # 组合关键字段
            fingerprint_text = '|'.join(str(field).strip() for field in key_fields)
            
            # 生成MD5哈希
            return hashlib.md5(fingerprint_text.encode('utf-8')).hexdigest()
            
        except Exception as e:
            self.logger.error(f"生成数据指纹失败: {e}")
            return hashlib.md5(str(datetime.now()).encode('utf-8')).hexdigest()
    
    def detect_duplicates(self, data_list: List[Dict[str, Any]]) -> Dict[str, Any]:
        """检测重复数据"""
        try:
            fingerprints = {}
            duplicates = []
            unique_data = []
            
            for i, item in enumerate(data_list):
                fingerprint = item.get('data_fingerprint')
                
                if fingerprint in fingerprints:
                    duplicates.append({
                        'index': i,
                        'fingerprint': fingerprint,
                        'original_index': fingerprints[fingerprint]
                    })
                else:
                    fingerprints[fingerprint] = i
                    unique_data.append(item)
            
            result = {
                'total_count': len(data_list),
                'unique_count': len(unique_data),
                'duplicate_count': len(duplicates),
                'unique_data': unique_data,
                'duplicates': duplicates
            }
            
            self.logger.info(f"去重完成: 总数 {result['total_count']}, 唯一 {result['unique_count']}, 重复 {result['duplicate_count']}")
            
            return result
            
        except Exception as e:
            self.logger.error(f"检测重复数据失败: {e}")
            return {
                'total_count': len(data_list),
                'unique_count': len(data_list),
                'duplicate_count': 0,
                'unique_data': data_list,
                'duplicates': []
            }
    
    def generate_quality_report(self, data_list: List[Dict[str, Any]]) -> Dict[str, Any]:
        """生成数据质量报告"""
        try:
            total_count = len(data_list)
            
            if total_count == 0:
                return {
                    'total_records': 0,
                    'valid_records': 0,
                    'quality_score': 0.0,
                    'field_completeness': {},
                    'issues': []
                }
            
            # 统计字段完整性
            field_stats = {
                'contact_person': 0,
                'contact_info': 0,
                'acceptable_rent': 0,
                'building_area': 0,
                'publish_date': 0
            }
            
            valid_records = 0
            issues = []
            
            for item in data_list:
                is_valid = True
                
                # 检查字段完整性
                for field in field_stats:
                    if item.get(field):
                        field_stats[field] += 1
                    else:
                        is_valid = False
                        issues.append(f"缺少字段: {field}")
                
                if is_valid:
                    valid_records += 1
            
            # 计算完整性百分比
            field_completeness = {
                field: (count / total_count) * 100
                for field, count in field_stats.items()
            }
            
            # 计算质量分数
            quality_score = (valid_records / total_count) * 100
            
            report = {
                'total_records': total_count,
                'valid_records': valid_records,
                'quality_score': round(quality_score, 2),
                'field_completeness': field_completeness,
                'issues': list(set(issues))  # 去重
            }
            
            self.logger.info(f"数据质量报告: 总数 {total_count}, 有效 {valid_records}, 质量分数 {quality_score:.2f}%")
            
            return report
            
        except Exception as e:
            self.logger.error(f"生成质量报告失败: {e}")
            return {
                'total_records': len(data_list),
                'valid_records': 0,
                'quality_score': 0.0,
                'field_completeness': {},
                'issues': [str(e)]
            }
